package cays.biquge;

import com.alibaba.fastjson.JSONPath;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import javax.print.Doc;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;

/**
 * 爬取笔趣阁小说的demo
 *
 * @author Chai yansheng
 * @create 2019-08-16 17:00
 **/
public class CrawlBiqugeTxtDemo {
    public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
            "(KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36";
    public static StringBuilder sb;
    /**
     * 将txt保存到本地文件夹
     * @param fileName
     * @param text
     */
    public void writeStringToFile(String fileName, String text) {
        // 若指定文件夹没有，则先创建
        String dirStr = "src\\main\\java\\cays\\biquge\\txt\\";
        File dir = new File(dirStr);
        if (!dir.exists()) {
            dir.mkdirs();
        }
        File file = new File(dirStr + fileName + ".txt");
        // 如果文件不存在则创建文件
        if (!file.exists()) {
            try {
                file.createNewFile();
            } catch (IOException e) {
                System.err.println("文件创建失败");
                e.printStackTrace();
            }
        }
        try {
            //true,则追加写入
            FileWriter resultFile = new FileWriter(file, true);
            PrintWriter myFile = new PrintWriter(resultFile);
            // 追加写入文件
            myFile.println(text);

            myFile.close();
            resultFile.close();
        } catch (IOException e) {
            System.err.println("写入文件失败");
            e.printStackTrace();
        }
    }

    /**
     * 获取网站document
     * @param url
     * @return
     */
    public Document getDocument(String url) {
        Document document = null;
        try {
            document = Jsoup.connect(url)
                    .header("User-Agent",USER_AGENT)
                    .timeout(10000).ignoreContentType(true).get();

        } catch (IOException e) {
            e.printStackTrace();
        }
        return document;
    }

    /**
     * 解析每一章节小说
     * @param url 章节url
     */
    public void parseChapter(String url) {
        Document document = getDocument(url);
        String chapterName, chapterContent;
        try {
            // 章节名称，www.xbiquge.la的标题select是 div.bookname > h1
            Elements bookName = document.select("div.bookname > h1");
            chapterName = bookName.text().trim();
            //System.out.println(chapterName);
            // 章节内容，www.xbiquge.la的内容select是 div#content
            Elements bookContent = document.select("div#content");
            chapterContent = bookContent.text().replace(" ", "\n");
            chapterContent = chapterName + "\n" + chapterContent + "\n";
            // 将章节名称和章节内容同时写入文件
            if (sb != null) {
                sb.append(chapterContent);
            }
            //writeStringToFile(fileName, chapterContent);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    /**
     * 获取动态界面
     * @param url
     * @return
     */
    public Document getJSDocument(String url) {
        final WebClient webClient = new WebClient(BrowserVersion.CHROME);//新建一个模拟谷歌Chrome浏览器的浏览器客户端对象

        webClient.getOptions().setThrowExceptionOnScriptError(false);//当JS执行出错的时候是否抛出异常, 这里选择不需要
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);//当HTTP的状态非200时是否抛出异常, 这里选择不需要
        webClient.getOptions().setActiveXNative(false);
        webClient.getOptions().setCssEnabled(false);//是否启用CSS, 因为不需要展现页面, 所以不需要启用
        webClient.getOptions().setJavaScriptEnabled(true); //很重要，启用JS
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());//很重要，设置支持AJAX
        HtmlPage page = null;
        try {
            page = webClient.getPage(url);//尝试加载上面图片例子给出的网页
        } catch (Exception e) {
            e.printStackTrace();
        }finally {
            webClient.close();
        }
        webClient.waitForBackgroundJavaScript(30000);//异步JS执行需要耗时,所以这里线程要阻塞30秒,等待异步JS执行结束
        String pageXml = page.asXml();//直接将加载完成的页面转换成xml格式的字符串
        //TODO 下面的代码就是对字符串的操作了,常规的爬虫操作,用到了比较好用的Jsoup库
        Document document = Jsoup.parse(pageXml);//获取html文档
        return document;
    }
    /**
     * 获取一本小说
     * @param fileName 要保存的小说名称
     * @param url 小说目录页面url
     */
    public void getText(String fileName, String url) {
        sb = new StringBuilder();
        Document document = getDocument(url);
        // 目录地址，www.xbiquge.la的目录地址的select是 div#list a
        Elements chapterUrls = document.select("div#list a");
        List<String> urls = new ArrayList<>();
        for (Element chapterUrl : chapterUrls) {
            urls.add(chapterUrl.attr("abs:href"));
        }
        // 遍历目录地址解析每一章节
        for (String urlStr : urls) {
            parseChapter(urlStr);
        }
        writeStringToFile(fileName, sb.toString());
        System.out.printf("完成");
    }
    public String getDouBanChapter(String url) {
        String chapter = "";
        return chapter;
    }
    /**
     * 获取Json数据体body
     * @param url json接口
     * @return
     */
    public String getResponseBody(String url) {
        String body = "";
        try {
            // Jsoup获取JSON页面数据失败 org.jsoup.UnsupportedMimeTypeException: Unhandled content type. Must be text/*, application/xml, or application/json
            // 调用 .ignoreContentType(true)
            Connection.Response response = Jsoup.connect(url)
                    .ignoreContentType(true)
                    .header("User-Agent",USER_AGENT)
                    .timeout(10000)
                    .execute();
            body = response.body();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return body;
    }
    public void getDouBanTxt(String fileName, String url) {
        sb = new StringBuilder();
        String baseUrl = "https://read.douban.com";
        String json = getResponseBody(url);
        List<String> relativeUrls = (List<String>) JSONPath.read(json, "$.list[*].links.reader");
        List<String> urls = new ArrayList<>();
        relativeUrls.forEach(relativeUrl -> {
            String chapterUrl = baseUrl + relativeUrl;
            urls.add(chapterUrl);
            System.out.println(chapterUrl);
        });
        String url1 = urls.get(0);
        Document document = getJSDocument(url1);
        try {
            TimeUnit.SECONDS.sleep(20);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        System.out.println("body :\r\n" + document.body());
        Elements titleElements = document.select("div.chapter-header");
        if (titleElements.size() == 0) {
            System.err.println("No title!");
            return;
        }
        String title = titleElements.get(0).text();
        Elements pElements = document.select("div.content p");
        StringBuilder content = new StringBuilder();
        content.append(title + "\n");
        pElements.forEach(p -> {
            Elements span = p.select("span");
            span.forEach(s -> {
                content.append(s.text());
            });
            content.append("/n");
        });
        System.out.println("title :" + title);
        System.out.println("content :" + content);
        //writeStringToFile(fileName, sb.toString());
        System.out.printf("完成");
    }
    public static void main(String[] args) {
        CrawlBiqugeTxtDemo biqugeTxtDemo = new CrawlBiqugeTxtDemo();
        String url;
        // 小说目录页面url
        // url = "http://www.xbiquge.la/10/10489/";
        // biqugeTxtDemo.getText("三寸人间", url);
        url = "https://read.douban.com/j/column_v2/32079294/chapters?start=0&limit=1000&latestFirst=0";
        biqugeTxtDemo.getDouBanTxt("天下刀宗", url);
    }
}